package com.example.crawler.task;

import com.example.crawler.entity.Chapter;
import com.example.crawler.entity.Content;
import com.example.crawler.entity.Novel;
import com.example.crawler.entity.NovelBO;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/**
 * FileName:  JobProcessor
 * Author:    REZ_VK
 * Date  :   2019-5-27 20:48
 * info :    这个是..
 */
@Component
public class NovelProcessor implements PageProcessor {

    private Logger logger = LoggerFactory.getLogger(NovelProcessor.class);

    @Autowired
    private NovelPipeline novelPipeline;

    /**
     * 起始url地址
     */
    private String url = "https://www.biquge5.com/shuku/";

    private Site site = Site.me()
            //设置编码
            // .setCharset("utf8")
            .setCharset("gbk")
            //设置超时时间
            .setTimeOut(10 * 1000)
            //设置重试间隔时间
            .setRetrySleepTime(3000)
            //设置重试次数
            .setRetryTimes(3)
            //设置用户代理（操作系统及版本、CPU 类型、浏览器及版本、浏览器渲染引擎、浏览器语言、浏览器插件等）
            .setUserAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Safari/537.36");

    @Override
    public void process(Page page) {
        //解析页面（获取小说名字，小说详情URL）
        List<Selectable> nodes = page.getHtml().css("ul.list-top div.col-md-5 a").nodes();
        if (nodes.size() != 0) {
            //如果集合不为空(小说列表页)
            nodes.forEach(item -> {
                //把获取的url地址放入任务队列（小说详情页）
                page.addTargetRequest(item.links().toString());
            });
            //将下一页放入任务队列（小说列表页）
            List<Selectable> list = page.getHtml().css("div.page a:nth-child(3) ").nodes();
            page.addTargetRequest(list.get(0).links().toString());
        } else {
            //如果集合为空(小说详情页)
            List<Selectable> chapters = page.getHtml().css("div#list a").nodes();
            chapters.forEach(item -> {
                //把获取的url地址放入任务队列(小说章节页)
                page.addTargetRequest(item.links().toString());
            });
            if (chapters.size() != 0) {
                //(小说详情页)
                String image = page.getHtml().css("div#fmimg img", "src").toString();
                String title = page.getHtml().css("h1", "text").toString();
                String autor = page.getHtml().css("div#info p:nth-child(2)", "text").toString();
                String description = page.getHtml().css("div#intro p", "text").toString();
                String category = page.getHtml().css("div.con_top>a:nth-of-type(2)", "text").toString();
                //封装保存
                Novel novel = new Novel();
                novel.setImage(image);
                novel.setTitle(title);
                novel.setAuthor(autor);
                novel.setDescription(description);
                novel.setCategory(category);
                page.putField("novel", novel);
                //List<Selectable> chapterTitle = page.getHtml().css("div#list a", "text").nodes();//章节信息
                //logger.info("断点");
            } else {
                //(小说章节页)
                String novelTitle = page.getHtml().css("div.con_top>a:nth-of-type(3)", "text").toString();
                String chapterTitle = page.getHtml().css("h1", "text").toString();
                String content = page.getHtml().css("div#content").toString();//html格式
                //String content = page.getHtml().css("div#content","text").toString();
                //封装保存
                Novel novel = new Novel();
                novel.setTitle(novelTitle);
                Chapter chapter = new Chapter();
                chapter.setTitle(chapterTitle);
                Content content1 = new Content();
                content1.setDetails(content);
                NovelBO novelBO = new NovelBO();
                novelBO.setChapter(chapter);
                novelBO.setContent(content1);
                novelBO.setNovel(novel);
                page.putField("novelBO", novelBO);
                //logger.info("断点");
            }
        }
    }

    @Override
    public Site getSite() {
        return site;
    }

    /**
     * initialDelay 当任务启动后，等待多久执行方法
     * fixedDelay 没隔多久执行一次
     */
    @Scheduled(initialDelay = 1000, fixedDelay = 60 * 60 * 1000)
    public void process() {
        long startTime, endTime;
        logger.info("开始爬取.....");
        startTime = System.currentTimeMillis();
        Spider.create(new NovelProcessor())
                //设置爬取url
                .addUrl(url)
                //设置内存和去重
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))
                //设置线程数
                .thread(20)
                //保存到MongoDB
                .addPipeline(novelPipeline)
                //开启
                .run();
        endTime = System.currentTimeMillis();
        logger.info("爬取结束.....耗时 : {}", ((endTime - startTime) / 1000));
    }
}
